{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 搜索最佳L1和L2正则参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Program Files\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = train['interest_level']\n",
    "X = train.drop(['interest_level'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Program Files\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((16286, 227), (16286,))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, train_size = 0.33,random_state = 666)\n",
    "X_train.shape,y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.1, 1, 2, 3], 'reg_lambda': [0.1, 1, 2, 3]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#设置L1和L2正则参数搜索范围\n",
    "reg_alpha = [0.1,1,2,3]    \n",
    "reg_lambda = [0.1,1,2,3]    \n",
    "\n",
    "param_test3_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, colsample_bylevel=0.7, colsample_bytree=0.6,\n",
       "       gamma=0, learning_rate=0.1, max_delta_step=0, max_depth=5,\n",
       "       min_child_weight=9, missing=None, n_estimators=270, nthread=-1,\n",
       "       objective='multi:softprob', reg_alpha=0, reg_lambda=1,\n",
       "       scale_pos_weight=1, seed=3, silent=True, subsample=0.7),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'reg_alpha': [0.1, 1, 2, 3], 'reg_lambda': [0.1, 1, 2, 3]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#根据2_1搜索结果设置max_depth和min_child_weight最优值\n",
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=270,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=9,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Program Files\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60551, std: 0.00737, params: {'reg_alpha': 0.1, 'reg_lambda': 0.1},\n",
       "  mean: -0.60579, std: 0.00754, params: {'reg_alpha': 0.1, 'reg_lambda': 1},\n",
       "  mean: -0.60512, std: 0.00742, params: {'reg_alpha': 0.1, 'reg_lambda': 2},\n",
       "  mean: -0.60531, std: 0.00717, params: {'reg_alpha': 0.1, 'reg_lambda': 3},\n",
       "  mean: -0.60464, std: 0.00840, params: {'reg_alpha': 1, 'reg_lambda': 0.1},\n",
       "  mean: -0.60472, std: 0.00802, params: {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  mean: -0.60406, std: 0.00709, params: {'reg_alpha': 1, 'reg_lambda': 2},\n",
       "  mean: -0.60408, std: 0.00643, params: {'reg_alpha': 1, 'reg_lambda': 3},\n",
       "  mean: -0.60435, std: 0.00805, params: {'reg_alpha': 2, 'reg_lambda': 0.1},\n",
       "  mean: -0.60373, std: 0.00813, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.60241, std: 0.00758, params: {'reg_alpha': 2, 'reg_lambda': 2},\n",
       "  mean: -0.60331, std: 0.00744, params: {'reg_alpha': 2, 'reg_lambda': 3},\n",
       "  mean: -0.60342, std: 0.00811, params: {'reg_alpha': 3, 'reg_lambda': 0.1},\n",
       "  mean: -0.60347, std: 0.00808, params: {'reg_alpha': 3, 'reg_lambda': 1},\n",
       "  mean: -0.60307, std: 0.00796, params: {'reg_alpha': 3, 'reg_lambda': 2},\n",
       "  mean: -0.60342, std: 0.00873, params: {'reg_alpha': 3, 'reg_lambda': 3}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 2},\n",
       " -0.60240858287722665)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_, gsearch3_1.best_score_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
